#coding=utf-8
import time
import traceback
import requests
from bs4 import BeautifulSoup
from pymongo import MongoClient

def extract_number_str(string):
    arr = [];
    for i in range(len(string)):
        if string[i].isalnum() or string[i] in ['.']:
            arr.append(string[i])
    result = "".join(arr)
    return result
#print(extract_number_str(u"-1.2"))


def get_share_holder_info(code):
    url = "http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_StockHolder/stockid/"+code+".phtml"
    #flow_holders_url = "http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CirculateStockHolder/stockid/"+code+".phtml"
    response = requests.request("get", url);
    response_content = response.content;
    response_content = response_content.decode("GBK").encode("utf-8")
    result = [];
    current_period = None
    soup = BeautifulSoup(response_content, "lxml", from_encoding="utf-8")
    trs = soup.select("#Table1 tr")
    for tr in trs:
        #print("--->" + str(len(tr.select("td"))))
        tds = tr.select("td")
        length = len(tds)
        if length == 0 or length == 1:
            current_period = {}
            result.append(current_period)
            continue
        if length == 2:
            key = tds[0].text
            if key == u'截至日期':
                key = u'截止日期'#同一转成 截止日期
            if key in [u'股东总数', u'平均持股数']:
                current_period[key] = extract_number_str(tds[1].text.strip())
            else:
                current_period[key] = tds[1].text.strip()

        if length == 5:
            if (tds[0].text == u'编号'):
                continue
            else:
                holder = {};
                holder['holder_name'] = tds[1].text.strip()
                holder['stock_count'] = float(extract_number_str(tds[2].text))#去掉↑之类的
                holder['percent_in_captcity'] = float(extract_number_str(tds[3].text))#去掉↑之类的
                holder['share_holder_type'] = tds[4].text.strip()
                if not current_period.has_key("holders"):
                    current_period['holders'] = []
                current_period['holders'].append(holder)
    return result


def get_flow_share_holder_info(code):
    url = "http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CirculateStockHolder/stockid/"+code+".phtml"
    response = requests.request("get", url);
    response_content = response.content;
    response_content = response_content.decode("GBK")
    result = [];
    current_period = None
    soup = BeautifulSoup(response_content, "lxml")
    trs = soup.select("#CirculateShareholderTable tr")
    for tr in trs:
        ##print("--->" + str(len(tr.select("td"))))
        tds = tr.select("td")
        length = len(tds)
        if length == 0 or length == 1:
            current_period = {}
            result.append(current_period)
            continue
        if length == 2:
            current_period[tds[0].text] = tds[1].text.strip()
        if length == 5:
            if (tds[0].text == u'编号'):
                continue
            else:
                holder = {};
                holder['holder_name'] = tds[1].text.strip()
                holder['stock_count'] = tds[2].text.strip()
                holder['percent_in_captcity'] = tds[3].text.strip()
                holder['share_holder_type'] = tds[4].text.strip()
                if not current_period.has_key("holders"):
                    current_period['holders'] = []
                current_period['holders'].append(holder)
    return result


def get_fund_share_holder_info(code):
    url = "http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_FundStockHolder/stockid/"+code+".phtml"
    response = requests.request("get", url);
    response_content = response.content;
    response_content = response_content.decode("GBK")
    result = [];
    current_period = None
    soup = BeautifulSoup(response_content, "lxml")
    trs = soup.select("#FundHoldSharesTable tr")
    for tr in trs:
        ##print("--->" + str(len(tr.select("td"))))
        tds = tr.select("td")
        length = len(tds)
        if length == 0 or length == 1:
            current_period = {}
            result.append(current_period)
            continue
        if length == 2:
            current_period[tds[0].text] = tds[1].text
        if length == 6:
            if (tds[0].text == u'基金名称'):
                continue
            else:
                holder = {};
                holder['holder_name'] = tds[0].text.strip()
                holder['stock_count'] = tds[2].text.strip()
                holder['percent_in_captcity'] = tds[3].text.strip()
                holder['share_holder_type'] = "基金"
                holder['fund_code'] = tds[1].text.strip() #基金代码
                holder['percent_in_fund_captcity'] = tds[5].text.strip()
                if not current_period.has_key("holders"):
                    current_period['holders'] = []
                current_period['holders'].append(holder)
    return result


def save_period_share_holder(code, category, period_datas):
    if period_datas and len(period_datas) > 0 :
        for period_data in period_datas:
            #过滤空的结果
            if len(period_data.items()) == 0: continue
            period_data['code'] = code
            period_data['category'] = category
            record = db['stock_period_share_holders'].find_one({ "code": code, "category": category, "截止日期": period_data[u'截止日期']})
            if record:
                continue
            db['stock_period_share_holders'].insert_one(period_data)
            holders = period_data['holders']
            if len(holders) > 0:
                for holder in holders:
                    period_data_denormal_item = period_data.copy()
                    del period_data_denormal_item['holders']
                    del period_data_denormal_item['_id']
                    period_data_denormal_item = dict(period_data_denormal_item.items() + holder.items())
                    ##todo: if it exist in db, it does not isert it
                    db['stock_period_share_holders_denormal'].insert_one(period_data_denormal_item)


mgClient = MongoClient('localhost', 27017)
db = mgClient['stockDataStore']

stockCodes = db['stock'].find({}, ["code"])
##stockCodes = [{'code': '600004'}]
for stockInfo in stockCodes:
    try:
        code = stockInfo['code']
        print("get the top 10 big " + code)
        time.sleep(3.5)
        result = get_share_holder_info(code)
        save_period_share_holder(code, "十大股东", result)

        time.sleep(3.5)
        result2 = get_flow_share_holder_info(code)
        save_period_share_holder(code, "十大流通股东", result2)

        time.sleep(3.5)
        result3 = get_fund_share_holder_info(code)
        save_period_share_holder(code, "基金持股", result3)
        print("finish get the top 10 big " + code)
    except Exception as es:
        traceback.print_exc()
        #print("catch exception: " + str(es))
        #raise es





